1 package edu.jiangxin.apktoolbox.convert.protobuf.unsupervised;
2
3 import com.google.protobuf.WireFormat;
4 import org.apache.commons.lang3.ArrayUtils;
5 import org.json.JSONObject;
6
7 import java.math.BigInteger;
8 import java.util.ArrayList;
9 import java.util.HashMap;
10 import java.util.List;
11 import java.util.Map;
12
13
14 public class ProtobufDecoder {
15
16 private static final String VALID_DATA = "valid_data";
17
18 private static final String LEFT_OVER_DATA = "left_over_data";
19
20 private static final String KEY_BYTE_RANGE = "byteRange";
21
22 private static final String KEY_FIELD_NUMBER = "fieldNumber";
23
24 private static final String KEY_TYPE = "type";
25
26 private static final String KEY_VALUE = "value";
27
28
29
30
31
32
33
34 public static String bytesDecoder(byte[] bytes) {
35 Map<String, Object> map = decodeProto(bytes);
36 @SuppressWarnings("unchecked")
37 List<Map<String, Object>> validItems = (List<Map<String, Object>>) map.get(VALID_DATA);
38 List<JSONObject> result = new ArrayList<>();
39 for (Map<String, Object> validItem : validItems) {
40 DecoderResult protobufPart = getProtobufPart(validItem);
41 JSONObject jsonObject = ProtobufUtils.getJsonObject(protobufPart);
42 result.add(jsonObject);
43 }
44 JSONObject jsonObject = new JSONObject();
45 jsonObject.put(VALID_DATA, result);
46 jsonObject.put(LEFT_OVER_DATA, map.get(LEFT_OVER_DATA));
47 return jsonObject.toString(2);
48 }
49
50
51
52
53
54
55
56 private static Map<String, Object> decodeProto(byte[] buffer) {
57 BufferReader reader = new BufferReader(buffer);
58 List<Object> parts = new ArrayList<>();
59 reader.trySkipGrpcHeader();
60
61 try {
62 while (reader.leftBytes() > 0) {
63 reader.checkpoint();
64
65 List<Integer> byteRange = new ArrayList<>();
66 byteRange.add(reader.getOffset());
67 int indexType = reader.readVarInt().intValue();
68 int type = indexType & 0b111;
69 int fieldNumber = indexType >> 3;
70
71 Object value;
72 if (type == WireFormat.WIRETYPE_VARINT) {
73 value = reader.readVarInt();
74 } else if (type == WireFormat.WIRETYPE_LENGTH_DELIMITED) {
75 int length = reader.readVarInt().intValue();
76 value = reader.readBuffer(length);
77 } else if (type == WireFormat.WIRETYPE_FIXED32) {
78 value = reader.readBuffer(4);
79 } else if (type == WireFormat.WIRETYPE_FIXED64) {
80 value = reader.readBuffer(8);
81 } else {
82 throw new RuntimeException("Unknown type: " + type);
83 }
84 byteRange.add(reader.getOffset());
85
86 Map<String, Object> part = new HashMap<>();
87 part.put(KEY_BYTE_RANGE, byteRange);
88 part.put(KEY_FIELD_NUMBER, fieldNumber);
89 part.put(KEY_TYPE, type);
90 part.put(KEY_VALUE, value);
91 parts.add(part);
92 }
93 } catch (Exception e) {
94 reader.resetToCheckpoint();
95 }
96
97 Map<String, Object> result = new HashMap<>();
98 result.put(VALID_DATA, parts);
99 result.put(LEFT_OVER_DATA, ByteUtil.bytesToHex(reader.readBuffer(reader.leftBytes())));
100
101 return result;
102 }
103
104
105
106
107
108
109
110 private static DecoderResult getProtobufPart(Map<String, Object> part) {
111 DecoderResult result = new DecoderResult();
112 @SuppressWarnings("unchecked")
113 List<Integer> byteRange = (List<Integer>) part.get(KEY_BYTE_RANGE);
114 result.setByteRange(byteRange);
115 result.setFieldNumber((int) part.get(KEY_FIELD_NUMBER));
116 int type = (int) part.get(KEY_TYPE);
117 Object value = part.get(KEY_VALUE);
118 switch (type) {
119 case WireFormat.WIRETYPE_VARINT:
120 String valueStr = value.toString();
121 result.setContent(decodeVarintParts(valueStr));
122 break;
123 case WireFormat.WIRETYPE_LENGTH_DELIMITED:
124 byte[] bytes = (byte[]) value;
125 Map<String, Object> decoded = decodeProto(bytes);
126 String leftOverData = (String) decoded.get(LEFT_OVER_DATA);
127 if (bytes != null && bytes.length > 0 && leftOverData != null && leftOverData.length() == 0) {
128 @SuppressWarnings("unchecked")
129 List<Map<String, Object>> decodedParts = (List<Map<String, Object>>) decoded.get(VALID_DATA);
130 List<DecoderResult> subResults = new ArrayList<>();
131 for (Map<String, Object> decodedPart : decodedParts) {
132 DecoderResult protobufPart = getProtobufPart(decodedPart);
133 subResults.add(protobufPart);
134 }
135 result.setSubResults(subResults);
136 } else {
137 Map<String, Object> map = decodeStringOrBytes(bytes);
138 result.setContent((String) map.get(KEY_VALUE));
139 }
140 break;
141 case WireFormat.WIRETYPE_FIXED64:
142 bytes = (byte[]) value;
143 List<Map<String, Object>> fixed64Result = decodeFixed64(bytes);
144 result.setContent(JSONObject.valueToString(fixed64Result));
145 break;
146 case WireFormat.WIRETYPE_FIXED32:
147 bytes = (byte[]) value;
148 List<Map<String, Object>> fixed32Result = decodeFixed32(bytes);
149 result.setContent(JSONObject.valueToString(fixed32Result));
150 break;
151 default:
152 break;
153 }
154 result.setType(type);
155 return result;
156 }
157
158
159
160
161
162
163 private static List<Map<String, Object>> decodeFixed32(byte[] value) {
164 float floatValue = ByteUtil.bytesToFloat(value);
165 int intValue = ByteUtil.bytesToInt(value);
166 int uintValue = ByteUtil.bytesToInt(value);
167
168 List<Map<String, Object>> result = new ArrayList<>(3);
169 Map<String, Object> map1 = new HashMap<>(2);
170 map1.put(KEY_TYPE, "Int");
171 map1.put(KEY_VALUE, intValue);
172 result.add(map1);
173
174 if (intValue != uintValue) {
175 Map<String, Object> map2 = new HashMap<>(2);
176 map2.put(KEY_TYPE, "Unsigned Int");
177 map2.put(KEY_VALUE, uintValue);
178 result.add(map2);
179 }
180 Map<String, Object> map3 = new HashMap<>(2);
181 map3.put(KEY_TYPE, "Float");
182 map3.put(KEY_VALUE, floatValue);
183 result.add(map3);
184 return result;
185 }
186
187
188
189
190
191
192 private static List<Map<String, Object>> decodeFixed64(byte[] value) {
193 double floatValue = ByteUtil.bytesToDouble(value);
194 BigInteger intValue = new BigInteger(ByteUtil.bytesToHex(value), 16);
195 BigInteger uintValue = twoComplements(intValue);
196
197 List<Map<String, Object>> result = new ArrayList<>(3);
198 Map<String, Object> map1 = new HashMap<>(2);
199 map1.put(KEY_TYPE, "Int");
200 map1.put(KEY_VALUE, intValue.toString());
201 result.add(map1);
202
203 if (!intValue.equals(uintValue)) {
204 Map<String, Object> map2 = new HashMap<>(2);
205 map2.put(KEY_TYPE, "Unsigned Int");
206 map2.put(KEY_VALUE, uintValue.toString());
207 result.add(map2);
208 }
209 Map<String, Object> map3 = new HashMap<>(2);
210 map3.put(KEY_TYPE, "Double");
211 map3.put(KEY_VALUE, floatValue);
212 result.add(map3);
213 return result;
214 }
215
216
217
218
219
220
221 private static Map<String, Object> decodeStringOrBytes(byte[] value) {
222 Map<String, Object> result = new HashMap<>(2);
223 if (ArrayUtils.isEmpty(value)) {
224 result.put(KEY_TYPE, "string|bytes");
225 result.put(KEY_VALUE, "");
226 return result;
227 }
228 try {
229 result.put(KEY_TYPE, "string");
230 result.put(KEY_VALUE, hexStrToStr(ByteUtil.bytesToHex(value), "utf-8"));
231 } catch (Exception e) {
232 result.put(KEY_TYPE, "bytes");
233 result.put(KEY_VALUE, ByteUtil.bytesToHex(value));
234 }
235 return result;
236 }
237
238
239
240
241
242
243 private static String decodeVarintParts(String value) {
244 List<Map<String, Object>> result = new ArrayList<>(3);
245 StringBuilder sb = new StringBuilder();
246 BigInteger intVal = new BigInteger(value);
247 Map<String, Object> map1 = new HashMap<>(2);
248 sb.append("[as uint:");
249 sb.append(intVal);
250 sb.append("]");
251 result.add(map1);
252
253 BigInteger signedIntVal = VarintUtils.interpretAsSignedType(intVal);
254 if (!signedIntVal.equals(intVal)) {
255 Map<String, Object> map2 = new HashMap<>(2);
256 sb.append("[as sint:");
257 sb.append(signedIntVal);
258 sb.append("]");
259 result.add(map2);
260 }
261 return sb.toString();
262 }
263
264
265
266
267
268
269
270
271 private static BigInteger twoComplements(BigInteger uintValue) {
272 if (uintValue.compareTo(new BigInteger("7fffffffffffffff", 16)) > 0) {
273 return uintValue.subtract(new BigInteger("10000000000000000", 16));
274 } else {
275 return uintValue;
276 }
277 }
278
279
280
281
282
283
284
285
286 private static String hexStrToStr(String string, String charsetName) {
287 if (string == null || string.equals("")) {
288 return null;
289 }
290 byte[] baKeyword = new byte[string.length() / 2];
291 for (int i = 0; i < baKeyword.length; i++) {
292 try {
293 baKeyword[i] = (byte) (0xff & Integer.parseInt(string.substring(i * 2, i * 2 + 2), 16));
294 } catch (Exception e) {
295 e.printStackTrace();
296 }
297 }
298 try {
299 string = new String(baKeyword, charsetName);
300 } catch (Exception e1) {
301 e1.printStackTrace();
302 }
303 return string;
304 }
305 }